func (k *Kafka) Connect() error {
config := sarama.NewConfig()
config.Producer.RequiredAcks = sarama.RequiredAcks(k.RequiredAcks)
config.Producer.Compression = sarama.CompressionCodec(k.CompressionCodec)
config.Producer.Retry.Max = k.MaxRetry
// Legacy support ssl config
if k.Certificate != "" {
k.SSLCert = k.Certificate
k.SSLCA = k.CA
k.SSLKey = k.Key
}
tlsConfig, err := internal.GetTLSConfig(
k.SSLCert, k.SSLKey, k.SSLCA, k.InsecureSkipVerify)
if err != nil {
return err
}
if tlsConfig != nil {
config.Net.TLS.Config = tlsConfig
config.Net.TLS.Enable = true
}
producer, err := sarama.NewSyncProducer(k.Brokers, config)
if err != nil {
return err
}
k.producer = producer
return nil
}
func newKafkaConfig(config *kafkaConfig) (*sarama.Config, error) {
k := sarama.NewConfig()
// configure network level properties
timeout := config.Timeout
k.Net.DialTimeout = timeout
k.Net.ReadTimeout = timeout
k.Net.WriteTimeout = timeout
k.Net.KeepAlive = config.KeepAlive
k.Producer.Timeout = config.BrokerTimeout
tls, err := outputs.LoadTLSConfig(config.TLS)
if err != nil {
return nil, err
}
k.Net.TLS.Enable = tls != nil
k.Net.TLS.Config = tls
// TODO: configure metadata level properties
// use lib defaults
// configure producer API properties
if config.MaxMessageBytes != nil {
k.Producer.MaxMessageBytes = *config.MaxMessageBytes
}
if config.RequiredACKs != nil {
k.Producer.RequiredAcks = sarama.RequiredAcks(*config.RequiredACKs)
}
compressionMode, ok := compressionModes[strings.ToLower(config.Compression)]
if !ok {
return nil, fmt.Errorf("Unknown compression mode: %v", config.Compression)
}
k.Producer.Compression = compressionMode
k.Producer.Return.Successes = true // enable return channel for signaling
k.Producer.Return.Errors = true
// have retries being handled by libbeat, disable retries in sarama library
retryMax := config.MaxRetries
if retryMax < 0 {
retryMax = 1000
}
k.Producer.Retry.Max = retryMax
// configure per broker go channel buffering
k.ChannelBufferSize = config.ChanBufferSize
// configure client ID
k.ClientID = config.ClientID
if err := k.Validate(); err != nil {
logp.Err("Invalid kafka configuration: %v", err)
return nil, err
}
return k, nil
}
func newKafkaConfig(config *kafkaConfig) (*sarama.Config, int, error) {
k := sarama.NewConfig()
modeRetries := 1
// configure network level properties
timeout := time.Duration(config.Timeout) * time.Second
k.Net.DialTimeout = timeout
k.Net.ReadTimeout = timeout
k.Net.WriteTimeout = timeout
k.Net.KeepAlive = config.KeepAlive
k.Producer.Timeout = config.BrokerTimeout
tls, err := outputs.LoadTLSConfig(config.TLS)
if err != nil {
return nil, modeRetries, err
}
k.Net.TLS.Enable = tls != nil
k.Net.TLS.Config = tls
// TODO: configure metadata level properties
// use lib defaults
// configure producer API properties
if config.MaxMessageBytes != nil {
k.Producer.MaxMessageBytes = *config.MaxMessageBytes
}
if config.RequiredACKs != nil {
k.Producer.RequiredAcks = sarama.RequiredAcks(*config.RequiredACKs)
}
compressionMode, ok := compressionModes[strings.ToLower(config.Compression)]
if !ok {
return nil, modeRetries, fmt.Errorf("Unknown compression mode: %v", config.Compression)
}
k.Producer.Compression = compressionMode
k.Producer.Return.Successes = true // enable return channel for signaling
k.Producer.Return.Errors = true
if config.MaxRetries != nil {
retries := *config.MaxRetries
if retries < 0 {
retries = 10
modeRetries = -1
}
k.Producer.Retry.Max = retries
}
// configure client ID
k.ClientID = config.ClientID
if err := k.Validate(); err != nil {
logp.Err("Invalid kafka configuration: %v", err)
return nil, modeRetries, err
}
return k, modeRetries, nil
}
func (this *MirrorMaker) startProducers() {
for i := 0; i < this.config.NumProducers; i++ {
conf, err := ProducerConfigFromFile(this.config.ProducerConfig)
if err != nil {
panic(err)
}
if err = conf.Validate(); err != nil {
panic(err)
}
client, err := sarama.NewClient(conf.Clientid, conf.BrokerList, sarama.NewClientConfig())
if err != nil {
panic(err)
}
config := sarama.NewProducerConfig()
config.ChannelBufferSize = conf.SendBufferSize
switch strings.ToLower(conf.CompressionCodec) {
case "none":
config.Compression = sarama.CompressionNone
case "gzip":
config.Compression = sarama.CompressionGZIP
case "snappy":
config.Compression = sarama.CompressionSnappy
}
config.FlushByteCount = conf.FlushByteCount
config.FlushFrequency = conf.FlushTimeout
config.FlushMsgCount = conf.BatchSize
config.MaxMessageBytes = conf.MaxMessageBytes
config.MaxMessagesPerReq = conf.MaxMessagesPerRequest
if this.config.PreservePartitions {
config.Partitioner = NewIntPartitioner
} else {
config.Partitioner = sarama.NewRandomPartitioner
}
config.RequiredAcks = sarama.RequiredAcks(conf.Acks)
config.RetryBackoff = conf.RetryBackoff
config.Timeout = conf.Timeout
producer, err := sarama.NewProducer(client, config)
if err != nil {
panic(err)
}
this.producers = append(this.producers, producer)
if this.config.PreserveOrder {
go this.produceRoutine(producer, i)
} else {
go this.produceRoutine(producer, 0)
}
}
}
func NewSaramaProducer(conf *ProducerConfig) Producer {
if err := conf.Validate(); err != nil {
panic(err)
}
client, err := sarama.NewClient(conf.Clientid, conf.BrokerList, sarama.NewClientConfig())
if err != nil {
panic(err)
}
config := sarama.NewProducerConfig()
config.ChannelBufferSize = conf.SendBufferSize
switch strings.ToLower(conf.CompressionCodec) {
case "none":
config.Compression = sarama.CompressionNone
case "gzip":
config.Compression = sarama.CompressionGZIP
case "snappy":
config.Compression = sarama.CompressionSnappy
}
config.FlushByteCount = conf.FlushByteCount
config.FlushFrequency = conf.FlushTimeout
config.FlushMsgCount = conf.BatchSize
config.MaxMessageBytes = conf.MaxMessageBytes
config.MaxMessagesPerReq = conf.MaxMessagesPerRequest
config.RequiredAcks = sarama.RequiredAcks(conf.Acks)
config.RetryBackoff = conf.RetryBackoff
config.Timeout = conf.Timeout
config.AckSuccesses = conf.AckSuccesses
partitionerFactory := &SaramaPartitionerFactory{conf.Partitioner}
config.Partitioner = partitionerFactory.PartitionerConstructor
producer, err := sarama.NewProducer(client, config)
if err != nil {
panic(err)
}
saramaProducer := &SaramaProducer{
saramaProducer: producer,
config: conf,
}
saramaProducer.initSuccesses()
saramaProducer.initErrors()
saramaProducer.initInput()
return saramaProducer
}
func (this *SyslogProducer) startProducers() {
brokerList := strings.Split(this.config.BrokerList, ",")
conf := this.config.ProducerConfig
config := sarama.NewConfig()
config.ClientID = conf.Clientid
config.ChannelBufferSize = conf.SendBufferSize
switch strings.ToLower(conf.CompressionCodec) {
case "none":
config.Producer.Compression = sarama.CompressionNone
case "gzip":
config.Producer.Compression = sarama.CompressionGZIP
case "snappy":
config.Producer.Compression = sarama.CompressionSnappy
}
config.Producer.Flush.Bytes = conf.FlushByteCount
config.Producer.Flush.Frequency = conf.FlushTimeout
config.Producer.Flush.Messages = conf.BatchSize
config.Producer.Flush.MaxMessages = conf.MaxMessagesPerRequest
config.Producer.MaxMessageBytes = conf.MaxMessageBytes
config.Producer.Partitioner = sarama.NewRandomPartitioner
config.Producer.RequiredAcks = sarama.RequiredAcks(conf.Acks)
config.Producer.Retry.Backoff = conf.RetryBackoff
config.Producer.Timeout = conf.Timeout
for i := 0; i < this.config.NumProducers; i++ {
client, err := sarama.NewClient(brokerList, config)
if err != nil {
panic(err)
}
Tracef(this, "Starting new producer with config: %#v", config)
producer, err := sarama.NewAsyncProducerFromClient(client)
if err != nil {
panic(err)
}
this.producers = append(this.producers, producer)
go this.produceRoutine(producer)
}
}
func newKafkaConfig(config *kafkaConfig) (*sarama.Config, error) {
k := sarama.NewConfig()
// configure network level properties
timeout := config.Timeout
k.Net.DialTimeout = timeout
k.Net.ReadTimeout = timeout
k.Net.WriteTimeout = timeout
k.Net.KeepAlive = config.KeepAlive
k.Producer.Timeout = config.BrokerTimeout
tls, err := outputs.LoadTLSConfig(config.TLS)
if err != nil {
return nil, err
}
if tls != nil {
k.Net.TLS.Enable = true
k.Net.TLS.Config = tls.BuildModuleConfig("")
}
if config.Username != "" {
k.Net.SASL.Enable = true
k.Net.SASL.User = config.Username
k.Net.SASL.Password = config.Password
}
// configure metadata update properties
k.Metadata.Retry.Max = config.Metadata.Retry.Max
k.Metadata.Retry.Backoff = config.Metadata.Retry.Backoff
k.Metadata.RefreshFrequency = config.Metadata.RefreshFreq
// configure producer API properties
if config.MaxMessageBytes != nil {
k.Producer.MaxMessageBytes = *config.MaxMessageBytes
}
if config.RequiredACKs != nil {
k.Producer.RequiredAcks = sarama.RequiredAcks(*config.RequiredACKs)
}
compressionMode, ok := compressionModes[strings.ToLower(config.Compression)]
if !ok {
return nil, fmt.Errorf("Unknown compression mode: '%v'", config.Compression)
}
k.Producer.Compression = compressionMode
k.Producer.Return.Successes = true // enable return channel for signaling
k.Producer.Return.Errors = true
// have retries being handled by libbeat, disable retries in sarama library
retryMax := config.MaxRetries
if retryMax < 0 {
retryMax = 1000
}
k.Producer.Retry.Max = retryMax
// TODO: k.Producer.Retry.Backoff = ?
// configure per broker go channel buffering
k.ChannelBufferSize = config.ChanBufferSize
// configure client ID
k.ClientID = config.ClientID
if err := k.Validate(); err != nil {
logp.Err("Invalid kafka configuration: %v", err)
return nil, err
}
version, ok := kafkaVersions[config.Version]
if !ok {
return nil, fmt.Errorf("Unknown/unsupported kafka version: %v", config.Version)
}
k.Version = version
return k, nil
}
// Configure initializes this producer with values from a plugin config.
func (prod *Kafka) Configure(conf core.PluginConfig) error {
err := prod.ProducerBase.Configure(conf)
if err != nil {
return err
}
prod.SetStopCallback(prod.close)
prod.servers = conf.GetStringArray("Servers", []string{"localhost:9092"})
prod.topic = conf.GetStreamMap("Topic", "")
prod.clientID = conf.GetString("ClientId", "gollum")
prod.lastMetricUpdate = time.Now()
prod.config = kafka.NewConfig()
prod.config.ClientID = conf.GetString("ClientId", "gollum")
prod.config.ChannelBufferSize = conf.GetInt("MessageBufferCount", 256)
prod.config.Net.MaxOpenRequests = conf.GetInt("MaxOpenRequests", 5)
prod.config.Net.DialTimeout = time.Duration(conf.GetInt("ServerTimeoutSec", 30)) * time.Second
prod.config.Net.ReadTimeout = prod.config.Net.DialTimeout
prod.config.Net.WriteTimeout = prod.config.Net.DialTimeout
prod.config.Metadata.Retry.Max = conf.GetInt("ElectRetries", 3)
prod.config.Metadata.Retry.Backoff = time.Duration(conf.GetInt("ElectTimeoutMs", 250)) * time.Millisecond
prod.config.Metadata.RefreshFrequency = time.Duration(conf.GetInt("MetadataRefreshMs", 10000)) * time.Millisecond
prod.config.Producer.MaxMessageBytes = conf.GetInt("BatchSizeMaxKB", 1<<10) << 10
prod.config.Producer.RequiredAcks = kafka.RequiredAcks(conf.GetInt("RequiredAcks", int(kafka.WaitForLocal)))
prod.config.Producer.Timeout = time.Duration(conf.GetInt("TimoutMs", 1500)) * time.Millisecond
prod.config.Producer.Return.Errors = true
prod.config.Producer.Return.Successes = true
switch strings.ToLower(conf.GetString("Compression", compressNone)) {
default:
fallthrough
case compressNone:
prod.config.Producer.Compression = kafka.CompressionNone
case compressGZIP:
prod.config.Producer.Compression = kafka.CompressionGZIP
case compressSnappy:
prod.config.Producer.Compression = kafka.CompressionSnappy
}
switch strings.ToLower(conf.GetString("Partitioner", partRandom)) {
case partRandom:
prod.config.Producer.Partitioner = kafka.NewRandomPartitioner
case partRoundrobin:
prod.config.Producer.Partitioner = kafka.NewRoundRobinPartitioner
default:
fallthrough
case partHash:
prod.config.Producer.Partitioner = kafka.NewHashPartitioner
}
prod.config.Producer.Flush.Bytes = conf.GetInt("BatchSizeByte", 8192)
prod.config.Producer.Flush.Messages = conf.GetInt("BatchMinCount", 1)
prod.config.Producer.Flush.Frequency = time.Duration(conf.GetInt("BatchTimeoutSec", 3)) * time.Second
prod.config.Producer.Flush.MaxMessages = conf.GetInt("BatchMaxCount", 0)
prod.config.Producer.Retry.Max = conf.GetInt("SendRetries", 3)
prod.config.Producer.Retry.Backoff = time.Duration(conf.GetInt("SendTimeoutMs", 100)) * time.Millisecond
prod.batch = core.NewMessageBatch(conf.GetInt("Channel", 8192))
prod.counters = make(map[string]*int64)
for _, topic := range prod.topic {
shared.Metric.New(kafkaMetricMessages + topic)
shared.Metric.New(kafkaMetricMessagesSec + topic)
prod.counters[topic] = new(int64)
}
shared.Metric.New(kafkaMetricMissCount)
prod.SetCheckFuseCallback(prod.tryOpenConnection)
return nil
}