Example #1
0
func initConnectionMode(
	cfg *common.Config,
	config *logstashConfig,
	transp *transport.Config,
) (mode.ConnectionMode, error) {
	sendRetries := config.MaxRetries
	maxAttempts := sendRetries + 1
	if sendRetries < 0 {
		maxAttempts = 0
	}

	settings := modeutil.Settings{
		Failover:     !config.LoadBalance,
		MaxAttempts:  maxAttempts,
		Timeout:      config.Timeout,
		WaitRetry:    defaultWaitRetry,
		MaxWaitRetry: defaultMaxWaitRetry,
	}

	if config.Pipelining == 0 {
		clients, err := modeutil.MakeClients(cfg, makeClientFactory(config, transp))
		if err != nil {
			return nil, err
		}
		return modeutil.NewConnectionMode(clients, settings)
	}

	clients, err := modeutil.MakeAsyncClients(cfg, makeAsyncClientFactory(config, transp))
	if err != nil {
		return nil, err
	}
	return modeutil.NewAsyncConnectionMode(clients, settings)
}
Example #2
0
func (lj *logstash) init(cfg *common.Config) error {
	config := defaultConfig
	if err := cfg.Unpack(&config); err != nil {
		return err
	}

	sendRetries := config.MaxRetries
	maxAttempts := sendRetries + 1
	if sendRetries < 0 {
		maxAttempts = 0
	}

	tls, err := outputs.LoadTLSConfig(config.TLS)
	if err != nil {
		return err
	}

	transp := &transport.Config{
		Timeout: config.Timeout,
		Proxy:   &config.Proxy,
		TLS:     tls,
		Stats: &transport.IOStats{
			Read:        statReadBytes,
			Write:       statWriteBytes,
			ReadErrors:  statReadErrors,
			WriteErrors: statWriteErrors,
		},
	}

	logp.Info("Max Retries set to: %v", sendRetries)
	var m mode.ConnectionMode
	if config.Pipelining == 0 {
		clients, err := modeutil.MakeClients(cfg, makeClientFactory(&config, transp))
		if err == nil {
			m, err = modeutil.NewConnectionMode(clients, !config.LoadBalance,
				maxAttempts, defaultWaitRetry, config.Timeout, defaultMaxWaitRetry)
		}
	} else {
		clients, err := modeutil.MakeAsyncClients(cfg,
			makeAsyncClientFactory(&config, transp))
		if err == nil {
			m, err = modeutil.NewAsyncConnectionMode(clients, !config.LoadBalance,
				maxAttempts, defaultWaitRetry, config.Timeout, defaultMaxWaitRetry)
		}
	}
	if err != nil {
		return err
	}

	lj.mode = m
	lj.index = config.Index

	return nil
}
Example #3
0
func (k *kafka) initMode(guaranteed bool) (mode.ConnectionMode, error) {
	libCfg, err := newKafkaConfig(&k.config)
	if err != nil {
		return nil, err
	}

	if guaranteed {
		libCfg.Producer.Retry.Max = 1000
	}

	worker := 1
	if k.config.Worker > 1 {
		worker = k.config.Worker
	}

	var clients []mode.AsyncProtocolClient
	hosts := k.config.Hosts
	topic := k.config.Topic
	useType := k.config.UseType
	for i := 0; i < worker; i++ {
		client, err := newKafkaClient(hosts, topic, useType, libCfg)
		if err != nil {
			logp.Err("Failed to create kafka client: %v", err)
			return nil, err
		}
		clients = append(clients, client)
	}

	maxAttempts := 1
	if guaranteed {
		maxAttempts = 0
	}

	mode, err := modeutil.NewAsyncConnectionMode(
		clients,
		false,
		maxAttempts,
		defaultWaitRetry,
		libCfg.Net.WriteTimeout,
		defaultMaxWaitRetry)
	if err != nil {
		logp.Err("Failed to configure kafka connection: %v", err)
		return nil, err
	}
	return mode, nil
}