func newSaramaBrokerConfig(config *ConsumerConfig) *sarama.BrokerConfig {
brokerConfig := sarama.NewBrokerConfig()
brokerConfig.DialTimeout = config.SocketTimeout
brokerConfig.ReadTimeout = config.SocketTimeout
brokerConfig.WriteTimeout = config.SocketTimeout
return brokerConfig
}
func (k *KafkaOutput) Init(config interface{}) (err error) {
k.config = config.(*KafkaOutputConfig)
if len(k.config.Addrs) == 0 {
return errors.New("addrs must have at least one entry")
}
k.cconfig = sarama.NewClientConfig()
k.cconfig.MetadataRetries = k.config.MetadataRetries
k.cconfig.WaitForElection = time.Duration(k.config.WaitForElection) * time.Millisecond
k.cconfig.BackgroundRefreshFrequency = time.Duration(k.config.BackgroundRefreshFrequency) * time.Millisecond
k.cconfig.DefaultBrokerConf = sarama.NewBrokerConfig()
k.cconfig.DefaultBrokerConf.MaxOpenRequests = k.config.MaxOpenRequests
k.cconfig.DefaultBrokerConf.DialTimeout = time.Duration(k.config.DialTimeout) * time.Millisecond
k.cconfig.DefaultBrokerConf.ReadTimeout = time.Duration(k.config.ReadTimeout) * time.Millisecond
k.cconfig.DefaultBrokerConf.WriteTimeout = time.Duration(k.config.WriteTimeout) * time.Millisecond
k.pconfig = sarama.NewProducerConfig()
switch k.config.Partitioner {
case "Random":
k.pconfig.Partitioner = sarama.NewRandomPartitioner()
if len(k.config.HashVariable) > 0 {
return fmt.Errorf("hash_variable should not be set for the %s partitioner", k.config.Partitioner)
}
case "RoundRobin":
k.pconfig.Partitioner = new(sarama.RoundRobinPartitioner)
if len(k.config.HashVariable) > 0 {
return fmt.Errorf("hash_variable should not be set for the %s partitioner", k.config.Partitioner)
}
case "Hash":
k.pconfig.Partitioner = sarama.NewHashPartitioner()
if k.hashVariable = verifyMessageVariable(k.config.HashVariable); k.hashVariable == nil {
return fmt.Errorf("invalid hash_variable: %s", k.config.HashVariable)
}
default:
return fmt.Errorf("invalid partitioner: %s", k.config.Partitioner)
}
if len(k.config.Topic) == 0 {
if k.topicVariable = verifyMessageVariable(k.config.TopicVariable); k.topicVariable == nil {
return fmt.Errorf("invalid topic_variable: %s", k.config.TopicVariable)
}
} else if len(k.config.TopicVariable) > 0 {
return errors.New("topic and topic_variable cannot both be set")
}
switch k.config.RequiredAcks {
case "NoResponse":
k.pconfig.RequiredAcks = sarama.NoResponse
case "WaitForLocal":
k.pconfig.RequiredAcks = sarama.WaitForLocal
case "WaitForAll":
k.pconfig.RequiredAcks = sarama.WaitForAll
default:
return fmt.Errorf("invalid required_acks: %s", k.config.RequiredAcks)
}
k.pconfig.Timeout = time.Duration(k.config.Timeout) * time.Millisecond
switch k.config.CompressionCodec {
case "None":
k.pconfig.Compression = sarama.CompressionNone
case "GZIP":
k.pconfig.Compression = sarama.CompressionGZIP
case "Snappy":
k.pconfig.Compression = sarama.CompressionSnappy
default:
return fmt.Errorf("invalid compression_codec: %s", k.config.CompressionCodec)
}
k.pconfig.MaxBufferedBytes = k.config.MaxBufferedBytes
k.pconfig.MaxBufferTime = time.Duration(k.config.MaxBufferTime) * time.Millisecond
k.pconfig.BackPressureThresholdBytes = k.config.BackPressureThresholdBytes
k.client, err = sarama.NewClient(k.config.Id, k.config.Addrs, k.cconfig)
if err != nil {
return
}
k.producer, err = sarama.NewProducer(k.client, k.pconfig)
return
}
func (k *KafkaInput) Init(config interface{}) (err error) {
k.config = config.(*KafkaInputConfig)
if len(k.config.Addrs) == 0 {
return errors.New("addrs must have at least one entry")
}
if len(k.config.Group) == 0 {
k.config.Group = k.config.Id
}
k.clientConfig = sarama.NewClientConfig()
k.clientConfig.MetadataRetries = k.config.MetadataRetries
k.clientConfig.WaitForElection = time.Duration(k.config.WaitForElection) * time.Millisecond
k.clientConfig.BackgroundRefreshFrequency = time.Duration(k.config.BackgroundRefreshFrequency) * time.Millisecond
k.clientConfig.DefaultBrokerConf = sarama.NewBrokerConfig()
k.clientConfig.DefaultBrokerConf.MaxOpenRequests = k.config.MaxOpenRequests
k.clientConfig.DefaultBrokerConf.DialTimeout = time.Duration(k.config.DialTimeout) * time.Millisecond
k.clientConfig.DefaultBrokerConf.ReadTimeout = time.Duration(k.config.ReadTimeout) * time.Millisecond
k.clientConfig.DefaultBrokerConf.WriteTimeout = time.Duration(k.config.WriteTimeout) * time.Millisecond
k.consumerConfig = sarama.NewConsumerConfig()
k.consumerConfig.DefaultFetchSize = k.config.DefaultFetchSize
k.consumerConfig.MinFetchSize = k.config.MinFetchSize
k.consumerConfig.MaxMessageSize = k.config.MaxMessageSize
k.consumerConfig.MaxWaitTime = time.Duration(k.config.MaxWaitTime) * time.Millisecond
k.checkpointFilename = k.pConfig.Globals.PrependBaseDir(filepath.Join("kafka",
fmt.Sprintf("%s.%s.%d.offset.bin", k.name, k.config.Topic, k.config.Partition)))
switch k.config.OffsetMethod {
case "Manual":
k.consumerConfig.OffsetMethod = sarama.OffsetMethodManual
if fileExists(k.checkpointFilename) {
if k.consumerConfig.OffsetValue, err = readCheckpoint(k.checkpointFilename); err != nil {
return fmt.Errorf("readCheckpoint %s", err)
}
} else {
if err = os.MkdirAll(filepath.Dir(k.checkpointFilename), 0766); err != nil {
return
}
k.consumerConfig.OffsetMethod = sarama.OffsetMethodOldest
}
case "Newest":
k.consumerConfig.OffsetMethod = sarama.OffsetMethodNewest
if fileExists(k.checkpointFilename) {
if err = os.Remove(k.checkpointFilename); err != nil {
return
}
}
case "Oldest":
k.consumerConfig.OffsetMethod = sarama.OffsetMethodOldest
if fileExists(k.checkpointFilename) {
if err = os.Remove(k.checkpointFilename); err != nil {
return
}
}
default:
return fmt.Errorf("invalid offset_method: %s", k.config.OffsetMethod)
}
k.consumerConfig.EventBufferSize = k.config.EventBufferSize
k.client, err = sarama.NewClient(k.config.Id, k.config.Addrs, k.clientConfig)
if err != nil {
return
}
k.consumer, err = sarama.NewConsumer(k.client, k.config.Topic, k.config.Partition, k.config.Group, k.consumerConfig)
return
}