// setupProducer returns a producer of kafka server
func setupProducer(sinkBrokerHosts []string, topic string, brokerConf kafka.BrokerConf) (kafka.DistributingProducer, error) {
glog.V(3).Infof("attempting to setup kafka sink")
broker, err := kafka.Dial(sinkBrokerHosts, brokerConf)
if err != nil {
return nil, fmt.Errorf("failed to connect to kafka cluster: %s", err)
}
defer broker.Close()
//create kafka producer
conf := kafka.NewProducerConf()
conf.RequiredAcks = proto.RequiredAcksLocal
producer := broker.Producer(conf)
// create RoundRobinProducer with the default producer.
count, err := broker.PartitionCount(topic)
if err != nil {
count = 1
glog.Warningf("Failed to get partition count of topic %q: %s", topic, err)
}
sinkProducer := kafka.NewRoundRobinProducer(producer, count)
glog.V(3).Infof("kafka sink setup successfully")
return sinkProducer, nil
}
func (self *KafkaOutput) Init(pcf *plugins.PluginCommonConfig, conf toml.Primitive) (err error) {
log.Println("KafkaOutput Init.")
self.common = pcf
hn, err := os.Hostname()
if err != nil {
hn = "kamanclient"
}
self.config = &KafkaOutputConfig{
ClientId: hn,
Distributer: "None",
Partitions: 0,
FlushInterval: 1000,
}
if err = toml.PrimitiveDecode(conf, self.config); err != nil {
return fmt.Errorf("Can't unmarshal KafkaOutput config: %s", err)
}
if len(self.config.Addrs) == 0 {
return errors.New("addrs must have at least one entry")
}
if len(self.config.Topic) == 0 {
return fmt.Errorf("topic is empty")
}
bcf := kafka.NewBrokerConf(self.config.ClientId)
//bcf.AllowTopicCreation = true
// connect to kafka cluster
self.broker, err = kafka.Dial(self.config.Addrs, bcf)
if err != nil {
return fmt.Errorf("cannot connect to kafka cluster: %s", err)
}
defer self.broker.Close()
pf := kafka.NewProducerConf()
pf.RequiredAcks = 1
self.producer = self.broker.Producer(pf)
partitions, err := self.broker.PartitionCount(self.config.Topic)
if err != nil {
return fmt.Errorf("cannot count to topic partitions: %s", err)
}
log.Printf("topic\"%s\" has %d partitions\n", self.config.Topic, partitions)
if (self.config.Partition + 1) > partitions {
return fmt.Errorf("invalid partition: %d, topic have %d partitions",
self.config.Partition, partitions)
}
if self.config.Partitions == 0 {
self.config.Partitions = partitions
}
switch self.config.Distributer {
case "Random":
self.distributingProducer = kafka.NewRandomProducer(self.producer, self.config.Partitions)
case "RoundRobin":
self.distributingProducer = kafka.NewRoundRobinProducer(self.producer, self.config.Partitions)
case "Hash":
self.distributingProducer = kafka.NewHashProducer(self.producer, self.config.Partitions)
case "None":
self.distributingProducer = nil
default:
return fmt.Errorf("invalid distributer: %s, must be one of these: \"Random\",\"RoundRobin\",\"Hash\"", self.config.Distributer)
}
self.batchChan = make(chan *outBatch)
self.backChan = make(chan *outBatch, 2) // Never block on the hand-back
return err
}