// setupProducer returns a producer of kafka server func setupProducer(sinkBrokerHosts []string, topic string, brokerConf kafka.BrokerConf) (kafka.DistributingProducer, error) { glog.V(3).Infof("attempting to setup kafka sink") broker, err := kafka.Dial(sinkBrokerHosts, brokerConf) if err != nil { return nil, fmt.Errorf("failed to connect to kafka cluster: %s", err) } defer broker.Close() //create kafka producer conf := kafka.NewProducerConf() conf.RequiredAcks = proto.RequiredAcksLocal producer := broker.Producer(conf) // create RoundRobinProducer with the default producer. count, err := broker.PartitionCount(topic) if err != nil { count = 1 glog.Warningf("Failed to get partition count of topic %q: %s", topic, err) } sinkProducer := kafka.NewRoundRobinProducer(producer, count) glog.V(3).Infof("kafka sink setup successfully") return sinkProducer, nil }
func (self *KafkaOutput) Init(pcf *plugins.PluginCommonConfig, conf toml.Primitive) (err error) { log.Println("KafkaOutput Init.") self.common = pcf hn, err := os.Hostname() if err != nil { hn = "kamanclient" } self.config = &KafkaOutputConfig{ ClientId: hn, Distributer: "None", Partitions: 0, FlushInterval: 1000, } if err = toml.PrimitiveDecode(conf, self.config); err != nil { return fmt.Errorf("Can't unmarshal KafkaOutput config: %s", err) } if len(self.config.Addrs) == 0 { return errors.New("addrs must have at least one entry") } if len(self.config.Topic) == 0 { return fmt.Errorf("topic is empty") } bcf := kafka.NewBrokerConf(self.config.ClientId) //bcf.AllowTopicCreation = true // connect to kafka cluster self.broker, err = kafka.Dial(self.config.Addrs, bcf) if err != nil { return fmt.Errorf("cannot connect to kafka cluster: %s", err) } defer self.broker.Close() pf := kafka.NewProducerConf() pf.RequiredAcks = 1 self.producer = self.broker.Producer(pf) partitions, err := self.broker.PartitionCount(self.config.Topic) if err != nil { return fmt.Errorf("cannot count to topic partitions: %s", err) } log.Printf("topic\"%s\" has %d partitions\n", self.config.Topic, partitions) if (self.config.Partition + 1) > partitions { return fmt.Errorf("invalid partition: %d, topic have %d partitions", self.config.Partition, partitions) } if self.config.Partitions == 0 { self.config.Partitions = partitions } switch self.config.Distributer { case "Random": self.distributingProducer = kafka.NewRandomProducer(self.producer, self.config.Partitions) case "RoundRobin": self.distributingProducer = kafka.NewRoundRobinProducer(self.producer, self.config.Partitions) case "Hash": self.distributingProducer = kafka.NewHashProducer(self.producer, self.config.Partitions) case "None": self.distributingProducer = nil default: return fmt.Errorf("invalid distributer: %s, must be one of these: \"Random\",\"RoundRobin\",\"Hash\"", self.config.Distributer) } self.batchChan = make(chan *outBatch) self.backChan = make(chan *outBatch, 2) // Never block on the hand-back return err }