func (lj *logstash) init(cfg *common.Config) error { config := defaultConfig if err := cfg.Unpack(&config); err != nil { return err } tls, err := outputs.LoadTLSConfig(config.TLS) if err != nil { return err } transp := &transport.Config{ Timeout: config.Timeout, Proxy: &config.Proxy, TLS: tls, Stats: &transport.IOStats{ Read: statReadBytes, Write: statWriteBytes, ReadErrors: statReadErrors, WriteErrors: statWriteErrors, }, } logp.Info("Max Retries set to: %v", config.MaxRetries) m, err := initConnectionMode(cfg, &config, transp) if err != nil { return err } lj.mode = m lj.index = config.Index return nil }
func newKafkaConfig(config *kafkaConfig) (*sarama.Config, int, error) { k := sarama.NewConfig() modeRetries := 1 // configure network level properties timeout := time.Duration(config.Timeout) * time.Second k.Net.DialTimeout = timeout k.Net.ReadTimeout = timeout k.Net.WriteTimeout = timeout k.Net.KeepAlive = config.KeepAlive k.Producer.Timeout = config.BrokerTimeout tls, err := outputs.LoadTLSConfig(config.TLS) if err != nil { return nil, modeRetries, err } k.Net.TLS.Enable = tls != nil k.Net.TLS.Config = tls // TODO: configure metadata level properties // use lib defaults // configure producer API properties if config.MaxMessageBytes != nil { k.Producer.MaxMessageBytes = *config.MaxMessageBytes } if config.RequiredACKs != nil { k.Producer.RequiredAcks = sarama.RequiredAcks(*config.RequiredACKs) } compressionMode, ok := compressionModes[strings.ToLower(config.Compression)] if !ok { return nil, modeRetries, fmt.Errorf("Unknown compression mode: %v", config.Compression) } k.Producer.Compression = compressionMode k.Producer.Return.Successes = true // enable return channel for signaling k.Producer.Return.Errors = true if config.MaxRetries != nil { retries := *config.MaxRetries if retries < 0 { retries = 10 modeRetries = -1 } k.Producer.Retry.Max = retries } // configure client ID k.ClientID = config.ClientID if err := k.Validate(); err != nil { logp.Err("Invalid kafka configuration: %v", err) return nil, modeRetries, err } return k, modeRetries, nil }
func newKafkaConfig(config *kafkaConfig) (*sarama.Config, error) { k := sarama.NewConfig() // configure network level properties timeout := config.Timeout k.Net.DialTimeout = timeout k.Net.ReadTimeout = timeout k.Net.WriteTimeout = timeout k.Net.KeepAlive = config.KeepAlive k.Producer.Timeout = config.BrokerTimeout tls, err := outputs.LoadTLSConfig(config.TLS) if err != nil { return nil, err } k.Net.TLS.Enable = tls != nil k.Net.TLS.Config = tls // TODO: configure metadata level properties // use lib defaults // configure producer API properties if config.MaxMessageBytes != nil { k.Producer.MaxMessageBytes = *config.MaxMessageBytes } if config.RequiredACKs != nil { k.Producer.RequiredAcks = sarama.RequiredAcks(*config.RequiredACKs) } compressionMode, ok := compressionModes[strings.ToLower(config.Compression)] if !ok { return nil, fmt.Errorf("Unknown compression mode: %v", config.Compression) } k.Producer.Compression = compressionMode k.Producer.Return.Successes = true // enable return channel for signaling k.Producer.Return.Errors = true // have retries being handled by libbeat, disable retries in sarama library retryMax := config.MaxRetries if retryMax < 0 { retryMax = 1000 } k.Producer.Retry.Max = retryMax // configure per broker go channel buffering k.ChannelBufferSize = config.ChanBufferSize // configure client ID k.ClientID = config.ClientID if err := k.Validate(); err != nil { logp.Err("Invalid kafka configuration: %v", err) return nil, err } return k, nil }
func (lj *logstash) init(cfg *common.Config) error { config := defaultConfig if err := cfg.Unpack(&config); err != nil { return err } sendRetries := config.MaxRetries maxAttempts := sendRetries + 1 if sendRetries < 0 { maxAttempts = 0 } tls, err := outputs.LoadTLSConfig(config.TLS) if err != nil { return err } transp := &transport.Config{ Timeout: config.Timeout, Proxy: &config.Proxy, TLS: tls, Stats: &transport.IOStats{ Read: statReadBytes, Write: statWriteBytes, ReadErrors: statReadErrors, WriteErrors: statWriteErrors, }, } logp.Info("Max Retries set to: %v", sendRetries) var m mode.ConnectionMode if config.Pipelining == 0 { clients, err := modeutil.MakeClients(cfg, makeClientFactory(&config, transp)) if err == nil { m, err = modeutil.NewConnectionMode(clients, !config.LoadBalance, maxAttempts, defaultWaitRetry, config.Timeout, defaultMaxWaitRetry) } } else { clients, err := modeutil.MakeAsyncClients(cfg, makeAsyncClientFactory(&config, transp)) if err == nil { m, err = modeutil.NewAsyncConnectionMode(clients, !config.LoadBalance, maxAttempts, defaultWaitRetry, config.Timeout, defaultMaxWaitRetry) } } if err != nil { return err } lj.mode = m lj.index = config.Index return nil }
func connectTLS(certName string) transportFactory { return func(addr string) (TransportClient, error) { tlsConfig, err := outputs.LoadTLSConfig(&outputs.TLSConfig{ CAs: []string{certName + ".pem"}, }) if err != nil { return nil, err } return newTLSClient(addr, 0, tlsConfig) } }
func create( info monitors.Info, cfg *common.Config, ) ([]monitors.Job, error) { config := DefaultConfig if err := cfg.Unpack(&config); err != nil { return nil, err } tls, err := outputs.LoadTLSConfig(config.TLS) if err != nil { return nil, err } defaultScheme := "tcp" if tls != nil { defaultScheme = "ssl" } addrs, err := collectHosts(&config, defaultScheme) if err != nil { return nil, err } if config.Socks5.URL != "" && !config.Socks5.LocalResolve { var jobs []monitors.Job for _, addr := range addrs { scheme, host := addr.Scheme, addr.Host for _, port := range addr.Ports { job, err := newTCPMonitorHostJob(scheme, host, port, tls, &config) if err != nil { return nil, err } jobs = append(jobs, job) } } return jobs, nil } jobs := make([]monitors.Job, len(addrs)) for i, addr := range addrs { jobs[i], err = newTCPMonitorIPsJob(addr, tls, &config) if err != nil { return nil, err } } return jobs, nil }
func connectTLS(timeout time.Duration, certName string) TransportFactory { return func(addr string, proxy *transport.ProxyConfig) (*transport.Client, error) { tlsConfig, err := outputs.LoadTLSConfig(&outputs.TLSConfig{ CAs: []string{certName + ".pem"}, }) if err != nil { return nil, err } cfg := transport.Config{ Proxy: proxy, TLS: tlsConfig, Timeout: timeout, } return transport.NewClient(&cfg, "tcp", addr, 0) } }
func (out *elasticsearchOutput) init( cfg *common.Config, topologyExpire int, ) error { config := defaultConfig if err := cfg.Unpack(&config); err != nil { return err } tlsConfig, err := outputs.LoadTLSConfig(config.TLS) if err != nil { return err } err = out.readTemplate(&config.Template) if err != nil { return err } clients, err := modeutil.MakeClients(cfg, makeClientFactory(tlsConfig, &config, out)) if err != nil { return err } maxRetries := config.MaxRetries maxAttempts := maxRetries + 1 // maximum number of send attempts (-1 = infinite) if maxRetries < 0 { maxAttempts = 0 } var waitRetry = time.Duration(1) * time.Second var maxWaitRetry = time.Duration(60) * time.Second out.clients = clients loadBalance := config.LoadBalance m, err := modeutil.NewConnectionMode(clients, !loadBalance, maxAttempts, waitRetry, config.Timeout, maxWaitRetry) if err != nil { return err } out.mode = m out.index = config.Index return nil }
func (lj *logstash) init(cfg *ucfg.Config) error { config := defaultConfig if err := cfg.Unpack(&config); err != nil { return err } useTLS := (config.TLS != nil) timeout := time.Duration(config.Timeout) * time.Second sendRetries := config.MaxRetries maxAttempts := sendRetries + 1 if sendRetries < 0 { maxAttempts = 0 } var clients []mode.ProtocolClient var err error if useTLS { var tlsConfig *tls.Config tlsConfig, err = outputs.LoadTLSConfig(config.TLS) if err != nil { return err } clients, err = mode.MakeClients(cfg, makeClientFactory(&config, makeTLSClient(config.Port, tlsConfig))) } else { clients, err = mode.MakeClients(cfg, makeClientFactory(&config, makeTCPClient(config.Port))) } if err != nil { return err } logp.Info("Max Retries set to: %v", sendRetries) m, err := mode.NewConnectionMode(clients, !config.LoadBalance, maxAttempts, waitRetry, timeout, maxWaitRetry) if err != nil { return err } lj.mode = m lj.index = config.Index return nil }
func NewMockServerTLS(t *testing.T, to time.Duration, cert string, proxy *transport.ProxyConfig) *MockServer { tcpListener, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatalf("failed to generate TCP listener") } tlsConfig, err := outputs.LoadTLSConfig(&outputs.TLSConfig{ Certificate: outputs.CertificateConfig{ Certificate: cert + ".pem", Key: cert + ".key", }, }) if err != nil { t.Fatalf("failed to load certificate") } listener := tls.NewListener(tcpListener, tlsConfig.BuildModuleConfig("")) server := &MockServer{Listener: listener, Timeout: to} server.Handshake = func(client net.Conn) { if server.Err != nil { return } server.ClientDeadline(client, server.Timeout) if server.Err != nil { return } tlsConn, ok := client.(*tls.Conn) if !ok { server.Err = errors.New("no tls connection") return } server.Err = tlsConn.Handshake() } server.Transp = func() (*transport.Client, error) { return connectTLS(to, cert)(server.Addr(), proxy) } return server }
func newMockServerTLS(t *testing.T, to time.Duration, cert string) *mockServer { tcpListener, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatalf("failed to generate TCP listener") } tlsConfig, err := outputs.LoadTLSConfig(&outputs.TLSConfig{ Certificate: cert + ".pem", CertificateKey: cert + ".key", }) if err != nil { t.Fatalf("failed to load certificate") } listener := tls.NewListener(tcpListener, tlsConfig) server := &mockServer{Listener: listener, timeout: to} server.handshake = func(client net.Conn) { if server.err != nil { return } server.clientDeadline(client, server.timeout) if server.err != nil { return } tlsConn, ok := client.(*tls.Conn) if !ok { server.err = errors.New("no tls connection") return } server.err = tlsConn.Handshake() } server.transp = func() (TransportClient, error) { return connectTLS(cert)(server.Addr()) } return server }
func (lj *logstash) init(cfg *common.Config) error { config := defaultConfig if err := cfg.Unpack(&config); err != nil { return err } sendRetries := config.MaxRetries maxAttempts := sendRetries + 1 if sendRetries < 0 { maxAttempts = 0 } tls, err := outputs.LoadTLSConfig(config.TLS) if err != nil { return err } transp := &transport.Config{ Timeout: config.Timeout, Proxy: &config.Proxy, TLS: tls, } clients, err := mode.MakeClients(cfg, makeClientFactory(&config, transp)) if err != nil { return err } logp.Info("Max Retries set to: %v", sendRetries) m, err := mode.NewConnectionMode(clients, !config.LoadBalance, maxAttempts, defaultWaitRetry, config.Timeout, defaultMaxWaitRetry) if err != nil { return err } lj.mode = m lj.index = config.Index return nil }
// New create a new instance of the partition MetricSet func New(base mb.BaseMetricSet) (mb.MetricSet, error) { config := defaultConfig if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } tls, err := outputs.LoadTLSConfig(config.TLS) if err != nil { return nil, err } cfg := sarama.NewConfig() cfg.Net.DialTimeout = base.Module().Config().Timeout cfg.Net.ReadTimeout = base.Module().Config().Timeout cfg.ClientID = config.ClientID cfg.Metadata.Retry.Max = config.Metadata.Retries cfg.Metadata.Retry.Backoff = config.Metadata.Backoff if tls != nil { cfg.Net.TLS.Enable = true cfg.Net.TLS.Config = tls.BuildModuleConfig("") } if config.Username != "" { cfg.Net.SASL.Enable = true cfg.Net.SASL.User = config.Username cfg.Net.SASL.Password = config.Password } broker := sarama.NewBroker(base.Host()) return &MetricSet{ BaseMetricSet: base, broker: broker, cfg: cfg, id: noID, topics: config.Topics, }, nil }
func (p *Poller) runOneTime() error { request := gorequest.New() url := p.config.Url method := p.config.Method switch method { case "get": request.Get(url) case "delete": request.Delete(url) case "head": request.Head(url) case "patch": request.Patch(url) case "post": request.Post(url) case "put": request.Put(url) default: return fmt.Errorf("Unsupported HTTP method %g", method) } // set timeout if p.config.Timeout != nil { request.Timeout(time.Duration(*p.config.Timeout) * time.Second) } else { request.Timeout(DefaultTimeout) } // set authentication if p.config.BasicAuth.Username != "" && p.config.BasicAuth.Password != "" { request.BasicAuth.Username = p.config.BasicAuth.Username request.BasicAuth.Password = p.config.BasicAuth.Password } // set tls config useTLS := (p.config.TLS != nil) if useTLS { var err error var tlsConfig *tls.Config tlsConfig, err = outputs.LoadTLSConfig(p.config.TLS) if err != nil { return err } request.TLSClientConfig(tlsConfig) } // set body if p.config.Body != "" { switch method { case "patch", "post", "put": request.SendString(p.config.Body) default: } } // set headers request.Header = p.config.Headers // set proxy if p.config.ProxyUrl != "" { request.Proxy(p.config.ProxyUrl) } logp.Debug("Httpbeat", "Executing HTTP request: %v", request) now := time.Now() resp, body, errs := request.End() if errs != nil { logp.Err("An error occured while executing HTTP request: %v", errs) return fmt.Errorf("An error occured while executing HTTP request: %v", errs) } requestEvent := Request{ Url: url, Method: method, Headers: p.config.Headers, Body: p.config.Body, } responseEvent := Response{ StatusCode: resp.StatusCode, Headers: p.GetResponseHeader(resp), Body: body, } event := HttpEvent{ ReadTime: now, DocumentType: p.config.DocumentType, Fields: p.config.Fields, Request: requestEvent, Response: responseEvent, } p.httpbeat.events.PublishEvent(event.ToMapStr()) return nil }
func (r *redisOut) init(cfg *common.Config, expireTopo int) error { config := defaultConfig if err := cfg.Unpack(&config); err != nil { return err } sendRetries := config.MaxRetries maxAttempts := config.MaxRetries + 1 if sendRetries < 0 { maxAttempts = 0 } var dataType redisDataType switch config.DataType { case "", "list": dataType = redisListType case "channel": dataType = redisChannelType default: return errors.New("Bad Redis data type") } index := []byte(config.Index) if len(index) == 0 { return fmt.Errorf("missing %v", cfg.PathOf("index")) } tls, err := outputs.LoadTLSConfig(config.TLS) if err != nil { return err } transp := &transport.Config{ Timeout: config.Timeout, Proxy: &config.Proxy, TLS: tls, Stats: &transport.IOStats{ Read: statReadBytes, Write: statWriteBytes, ReadErrors: statReadErrors, WriteErrors: statWriteErrors, }, } // configure topology support r.topology.init(transp, topoConfig{ host: config.HostTopology, password: config.PasswordTopology, db: config.DbTopology, expire: time.Duration(expireTopo) * time.Second, }) // configure publisher clients clients, err := modeutil.MakeClients(cfg, func(host string) (mode.ProtocolClient, error) { t, err := transport.NewClient(transp, "tcp", host, config.Port) if err != nil { return nil, err } return newClient(t, config.Password, config.Db, index, dataType), nil }) if err != nil { return err } logp.Info("Max Retries set to: %v", sendRetries) m, err := modeutil.NewConnectionMode(clients, !config.LoadBalance, maxAttempts, defaultWaitRetry, config.Timeout, defaultMaxWaitRetry) if err != nil { return err } r.mode = m return nil }
func (out *elasticsearchOutput) init( cfg *ucfg.Config, topologyExpire int, ) error { config := defaultConfig if err := cfg.Unpack(&config); err != nil { return err } tlsConfig, err := outputs.LoadTLSConfig(config.TLS) if err != nil { return err } clients, err := mode.MakeClients(cfg, makeClientFactory(tlsConfig, &config)) if err != nil { return err } timeout := time.Duration(config.Timeout) * time.Second maxRetries := config.MaxRetries maxAttempts := maxRetries + 1 // maximum number of send attempts (-1 = infinite) if maxRetries < 0 { maxAttempts = 0 } var waitRetry = time.Duration(1) * time.Second var maxWaitRetry = time.Duration(60) * time.Second out.clients = clients loadBalance := config.LoadBalance m, err := mode.NewConnectionMode(clients, !loadBalance, maxAttempts, waitRetry, timeout, maxWaitRetry) if err != nil { return err } loadTemplate(config.Template, clients) if config.SaveTopology { err := out.EnableTTL() if err != nil { logp.Err("Fail to set _ttl mapping: %s", err) // keep trying in the background go func() { for { err := out.EnableTTL() if err == nil { break } logp.Err("Fail to set _ttl mapping: %s", err) time.Sleep(5 * time.Second) } }() } } out.TopologyExpire = 15000 if topologyExpire != 0 { out.TopologyExpire = topologyExpire * 1000 // millisec } out.mode = m out.index = config.Index return nil }
func (r *redisOut) init(cfg *common.Config, expireTopo int) error { config := defaultConfig if err := cfg.Unpack(&config); err != nil { return err } sendRetries := config.MaxRetries maxAttempts := config.MaxRetries + 1 if sendRetries < 0 { maxAttempts = 0 } var dataType redisDataType switch config.DataType { case "", "list": dataType = redisListType case "channel": dataType = redisChannelType default: return errors.New("Bad Redis data type") } if cfg.HasField("index") && !cfg.HasField("key") { s, err := cfg.String("index", -1) if err != nil { return err } if err := cfg.SetString("key", -1, s); err != nil { return err } } if !cfg.HasField("key") { cfg.SetString("key", -1, r.beatName) } key, err := outil.BuildSelectorFromConfig(cfg, outil.Settings{ Key: "key", MultiKey: "keys", EnableSingleOnly: true, FailEmpty: true, }) if err != nil { return err } tls, err := outputs.LoadTLSConfig(config.TLS) if err != nil { return err } transp := &transport.Config{ Timeout: config.Timeout, Proxy: &config.Proxy, TLS: tls, Stats: &transport.IOStats{ Read: statReadBytes, Write: statWriteBytes, ReadErrors: statReadErrors, WriteErrors: statWriteErrors, }, } // configure topology support r.topology.init(transp, topoConfig{ host: config.HostTopology, password: config.PasswordTopology, db: config.DbTopology, expire: time.Duration(expireTopo) * time.Second, }) // configure publisher clients clients, err := modeutil.MakeClients(cfg, func(host string) (mode.ProtocolClient, error) { t, err := transport.NewClient(transp, "tcp", host, config.Port) if err != nil { return nil, err } return newClient(t, config.Password, config.Db, key, dataType), nil }) if err != nil { return err } logp.Info("Max Retries set to: %v", sendRetries) m, err := modeutil.NewConnectionMode(clients, modeutil.Settings{ Failover: !config.LoadBalance, MaxAttempts: maxAttempts, Timeout: config.Timeout, WaitRetry: defaultWaitRetry, MaxWaitRetry: defaultMaxWaitRetry, }) if err != nil { return err } r.mode = m return nil }
func newKafkaConfig(config *kafkaConfig) (*sarama.Config, error) { k := sarama.NewConfig() // configure network level properties timeout := config.Timeout k.Net.DialTimeout = timeout k.Net.ReadTimeout = timeout k.Net.WriteTimeout = timeout k.Net.KeepAlive = config.KeepAlive k.Producer.Timeout = config.BrokerTimeout tls, err := outputs.LoadTLSConfig(config.TLS) if err != nil { return nil, err } if tls != nil { k.Net.TLS.Enable = true k.Net.TLS.Config = tls.BuildModuleConfig("") } if config.Username != "" { k.Net.SASL.Enable = true k.Net.SASL.User = config.Username k.Net.SASL.Password = config.Password } // configure metadata update properties k.Metadata.Retry.Max = config.Metadata.Retry.Max k.Metadata.Retry.Backoff = config.Metadata.Retry.Backoff k.Metadata.RefreshFrequency = config.Metadata.RefreshFreq // configure producer API properties if config.MaxMessageBytes != nil { k.Producer.MaxMessageBytes = *config.MaxMessageBytes } if config.RequiredACKs != nil { k.Producer.RequiredAcks = sarama.RequiredAcks(*config.RequiredACKs) } compressionMode, ok := compressionModes[strings.ToLower(config.Compression)] if !ok { return nil, fmt.Errorf("Unknown compression mode: '%v'", config.Compression) } k.Producer.Compression = compressionMode k.Producer.Return.Successes = true // enable return channel for signaling k.Producer.Return.Errors = true // have retries being handled by libbeat, disable retries in sarama library retryMax := config.MaxRetries if retryMax < 0 { retryMax = 1000 } k.Producer.Retry.Max = retryMax // TODO: k.Producer.Retry.Backoff = ? // configure per broker go channel buffering k.ChannelBufferSize = config.ChanBufferSize // configure client ID k.ClientID = config.ClientID if err := k.Validate(); err != nil { logp.Err("Invalid kafka configuration: %v", err) return nil, err } version, ok := kafkaVersions[config.Version] if !ok { return nil, fmt.Errorf("Unknown/unsupported kafka version: %v", config.Version) } k.Version = version return k, nil }
func create( info monitors.Info, cfg *common.Config, ) ([]monitors.Job, error) { config := defaultConfig if err := cfg.Unpack(&config); err != nil { return nil, err } tls, err := outputs.LoadTLSConfig(config.TLS) if err != nil { return nil, err } var body []byte var enc contentEncoder if config.Check.SendBody != "" { var err error compression := config.Check.Compression enc, err = getContentEncoder(compression.Type, compression.Level) if err != nil { return nil, err } buf := bytes.NewBuffer(nil) err = enc.Encode(buf, bytes.NewBufferString(config.Check.SendBody)) if err != nil { return nil, err } body = buf.Bytes() } validator := makeValidateResponse(&config.Check) jobs := make([]monitors.Job, len(config.URLs)) if config.ProxyURL != "" { transport, err := newRoundTripper(&config, tls) if err != nil { return nil, err } for i, url := range config.URLs { jobs[i], err = newHTTPMonitorHostJob(url, &config, transport, enc, body, validator) if err != nil { return nil, err } } } else { for i, url := range config.URLs { jobs[i], err = newHTTPMonitorIPsJob(&config, url, tls, enc, body, validator) if err != nil { return nil, err } } } return jobs, nil }
func (lj *logstash) init( config outputs.MothershipConfig, topologyExpire int, ) error { useTLS := (config.TLS != nil) timeout := logstashDefaultTimeout if config.Timeout != 0 { timeout = time.Duration(config.Timeout) * time.Second } defaultPort := logstashDefaultPort if config.Port != 0 { defaultPort = config.Port } maxWindowSize := defaultMaxWindowSize if config.BulkMaxSize != nil { maxWindowSize = *config.BulkMaxSize } compressLevel := defaultCompressionLevel if config.CompressionLevel != nil { compressLevel = *config.CompressionLevel } var clients []mode.ProtocolClient var err error if useTLS { var tlsConfig *tls.Config tlsConfig, err = outputs.LoadTLSConfig(config.TLS) if err != nil { return err } clients, err = mode.MakeClients(config, makeClientFactory(maxWindowSize, compressLevel, timeout, makeTLSClient(defaultPort, tlsConfig))) } else { clients, err = mode.MakeClients(config, makeClientFactory(maxWindowSize, compressLevel, timeout, makeTCPClient(defaultPort))) } if err != nil { return err } sendRetries := defaultSendRetries if config.MaxRetries != nil { sendRetries = *config.MaxRetries } logp.Info("Max Retries set to: %v", sendRetries) maxAttempts := sendRetries + 1 if sendRetries < 0 { maxAttempts = 0 } loadBalance := config.LoadBalance != nil && *config.LoadBalance m, err := mode.NewConnectionMode(clients, !loadBalance, maxAttempts, waitRetry, timeout, maxWaitRetry) if err != nil { return err } lj.mode = m lj.index = config.Index return nil }
func (p *Poller) runOneTime() error { if p.request == nil { p.request = gorequest.New() } url := p.config.Url method := p.config.Method switch method { case "get": p.request.Get(url) case "delete": p.request.Delete(url) case "head": p.request.Head(url) case "patch": p.request.Patch(url) case "post": p.request.Post(url) case "put": p.request.Put(url) default: return fmt.Errorf("Unsupported HTTP method %g", method) } outputFormat := p.config.OutputFormat switch outputFormat { case "": outputFormat = config.DefaultOutputFormat case "string": case "json": break default: return fmt.Errorf("Unsupported output format %g", outputFormat) } // set timeout if p.config.Timeout != nil { p.request.Timeout(time.Duration(*p.config.Timeout) * time.Second) } else { p.request.Timeout(config.DefaultTimeout) } // set authentication if p.config.BasicAuth.Username != "" && p.config.BasicAuth.Password != "" { p.request.BasicAuth.Username = p.config.BasicAuth.Username p.request.BasicAuth.Password = p.config.BasicAuth.Password } // set tls config useTLS := (p.config.SSL != nil) if useTLS { var err error var tlsConfig *tls.Config var tlsC *transport.TLSConfig //tlsConfig, err = outputs.LoadTLSConfig(p.config.TLS) tlsC, err = outputs.LoadTLSConfig(p.config.SSL) tlsConfig = convertTLSConfig(tlsC) if err != nil { return err } p.request.TLSClientConfig(tlsConfig) } // set body if p.config.Body != "" { switch method { case "patch", "post", "put": p.request.SendString(p.config.Body) default: } } // set headers p.request.Header = p.config.Headers // set proxy if p.config.ProxyUrl != "" { p.request.Proxy(p.config.ProxyUrl) } logp.Debug("Httpbeat", "Executing HTTP request: %v", p.request) now := time.Now() resp, body, errs := p.request.End() if errs != nil { p.request = nil logp.Err("An error occurred while executing HTTP request: %v", errs) return fmt.Errorf("An error occurred while executing HTTP request: %v", errs) } requestEvent := Request{ Url: url, Method: method, Headers: p.config.Headers, Body: p.config.Body, } var jsonBody map[string]interface{} responseEvent := Response{ StatusCode: resp.StatusCode, Headers: p.GetResponseHeader(resp), } if outputFormat == "string" { responseEvent.Body = body } else { if outputFormat == "json" { decoder := json.NewDecoder(strings.NewReader(body)) decoder.UseNumber() errs := decoder.Decode(&jsonBody) if errs != nil { jsonBody = nil logp.Err("An error occurred while marshalling response to JSON: %w", errs) } else { if p.config.JsonDotMode == "unflatten" { jsonBody = unflat(jsonBody).(map[string]interface{}) } else if p.config.JsonDotMode == "replace" { jsonBody = replaceDots(jsonBody).(map[string]interface{}) } } responseEvent.JsonBody = jsonBody } } event := HttpEvent{ ReadTime: now, DocumentType: p.config.DocumentType, Fields: p.config.Fields, Request: requestEvent, Response: responseEvent, } p.httpbeat.client.PublishEvent(event.ToMapStr()) return nil }
func (out *elasticsearchOutput) init( config outputs.MothershipConfig, topologyExpire int, ) error { tlsConfig, err := outputs.LoadTLSConfig(config.TLS) if err != nil { return err } clients, err := mode.MakeClients(config, makeClientFactory(tlsConfig, config)) if err != nil { return err } timeout := elasticsearchDefaultTimeout if config.Timeout != 0 { timeout = time.Duration(config.Timeout) * time.Second } maxRetries := defaultMaxRetries if config.MaxRetries != nil { maxRetries = *config.MaxRetries } maxAttempts := maxRetries + 1 // maximum number of send attempts (-1 = infinite) if maxRetries < 0 { maxAttempts = 0 } var waitRetry = time.Duration(1) * time.Second var maxWaitRetry = time.Duration(60) * time.Second var m mode.ConnectionMode out.clients = clients if len(clients) == 1 { client := clients[0] m, err = mode.NewSingleConnectionMode(client, maxAttempts, waitRetry, timeout, maxWaitRetry) } else { loadBalance := config.LoadBalance == nil || *config.LoadBalance if loadBalance { m, err = mode.NewLoadBalancerMode(clients, maxAttempts, waitRetry, timeout, maxWaitRetry) } else { m, err = mode.NewFailOverConnectionMode(clients, maxAttempts, waitRetry, timeout) } } if err != nil { return err } if config.Save_topology { err := out.EnableTTL() if err != nil { logp.Err("Fail to set _ttl mapping: %s", err) // keep trying in the background go func() { for { err := out.EnableTTL() if err == nil { break } logp.Err("Fail to set _ttl mapping: %s", err) time.Sleep(5 * time.Second) } }() } } out.TopologyExpire = 15000 if topologyExpire != 0 { out.TopologyExpire = topologyExpire * 1000 // millisec } out.mode = m out.index = config.Index return nil }
func (out *elasticsearchOutput) init( cfg *common.Config, topologyExpire int, ) error { config := defaultConfig if err := cfg.Unpack(&config); err != nil { return err } index, err := outil.BuildSelectorFromConfig(cfg, outil.Settings{ Key: "index", MultiKey: "indices", EnableSingleOnly: true, FailEmpty: true, }) if err != nil { return err } tlsConfig, err := outputs.LoadTLSConfig(config.TLS) if err != nil { return err } err = out.readTemplate(&config.Template) if err != nil { return err } out.index = index pipeline, err := outil.BuildSelectorFromConfig(cfg, outil.Settings{ Key: "pipeline", MultiKey: "pipelines", EnableSingleOnly: true, FailEmpty: false, }) if err != nil { return err } if !pipeline.IsEmpty() { out.pipeline = &pipeline } clients, err := modeutil.MakeClients(cfg, makeClientFactory(tlsConfig, &config, out)) if err != nil { return err } maxRetries := config.MaxRetries maxAttempts := maxRetries + 1 // maximum number of send attempts (-1 = infinite) if maxRetries < 0 { maxAttempts = 0 } var waitRetry = time.Duration(1) * time.Second var maxWaitRetry = time.Duration(60) * time.Second out.clients = clients loadBalance := config.LoadBalance m, err := modeutil.NewConnectionMode(clients, modeutil.Settings{ Failover: !loadBalance, MaxAttempts: maxAttempts, Timeout: config.Timeout, WaitRetry: waitRetry, MaxWaitRetry: maxWaitRetry, }) if err != nil { return err } out.mode = m return nil }
func New() (*Importer, error) { importer := Importer{} /* define the command line arguments */ cl, err := DefineCommandLine() if err != nil { cl.flagSet.Usage() return nil, err } /* parse command line arguments */ err = cl.ParseCommandLine() if err != nil { return nil, err } importer.cl = cl /* prepare the Elasticsearch index pattern */ fmtstr, err := fmtstr.CompileEvent(cl.opt.Index) if err != nil { return nil, fmt.Errorf("fail to build the Elasticsearch index pattern: %s", err) } indexSel := outil.MakeSelector(outil.FmtSelectorExpr(fmtstr, "")) var tlsConfig outputs.TLSConfig var tls *transport.TLSConfig if cl.opt.Insecure { tlsConfig.VerificationMode = transport.VerifyNone } if len(cl.opt.Certificate) > 0 && len(cl.opt.CertificateKey) > 0 { tlsConfig.Certificate = outputs.CertificateConfig{ Certificate: cl.opt.Certificate, Key: cl.opt.CertificateKey, } } if len(cl.opt.CertificateAuthority) > 0 { tlsConfig.CAs = []string{cl.opt.CertificateAuthority} } tls, err = outputs.LoadTLSConfig(&tlsConfig) if err != nil { return nil, fmt.Errorf("fail to load the SSL certificate: %s", err) } /* connect to Elasticsearch */ client, err := elasticsearch.NewClient( elasticsearch.ClientSettings{ URL: cl.opt.ES, Index: indexSel, TLS: tls, Username: cl.opt.User, Password: cl.opt.Pass, Timeout: 60 * time.Second, }, nil, ) if err != nil { return nil, fmt.Errorf("fail to connect to Elasticsearch: %s", err) } importer.client = client return &importer, nil }
func (lj *logstash) init( config outputs.MothershipConfig, topologyExpire int, ) error { useTLS := (config.TLS != nil) timeout := logstashDefaultTimeout if config.Timeout != 0 { timeout = time.Duration(config.Timeout) * time.Second } defaultPort := logstashDefaultPort if config.Port != 0 { defaultPort = config.Port } maxWindowSize := defaultMaxWindowSize if config.BulkMaxSize != nil { maxWindowSize = *config.BulkMaxSize } var clients []mode.ProtocolClient var err error if useTLS { var tlsConfig *tls.Config tlsConfig, err = outputs.LoadTLSConfig(config.TLS) if err != nil { return err } clients, err = mode.MakeClients(config, makeClientFactory(maxWindowSize, timeout, func(host string) (TransportClient, error) { return newTLSClient(host, defaultPort, tlsConfig) })) } else { clients, err = mode.MakeClients(config, makeClientFactory(maxWindowSize, timeout, func(host string) (TransportClient, error) { return newTCPClient(host, defaultPort) })) } if err != nil { return err } sendRetries := defaultSendRetries if config.MaxRetries != nil { sendRetries = *config.MaxRetries } logp.Info("Max Retries set to: %v", sendRetries) maxAttempts := sendRetries + 1 if sendRetries < 0 { maxAttempts = 0 } var m mode.ConnectionMode if len(clients) == 1 { m, err = mode.NewSingleConnectionMode(clients[0], maxAttempts, waitRetry, timeout, maxWaitRetry) } else { loadBalance := config.LoadBalance != nil && *config.LoadBalance if loadBalance { m, err = mode.NewLoadBalancerMode(clients, maxAttempts, waitRetry, timeout, maxWaitRetry) } else { m, err = mode.NewFailOverConnectionMode(clients, maxAttempts, waitRetry, timeout) } } if err != nil { return err } lj.mode = m lj.index = config.Index return nil }