Ejemplo n.º 1
0
// setupProducer returns a producer of kafka server
func setupProducer(sinkBrokerHosts []string, topic string, brokerConf kafka.BrokerConf) (kafka.DistributingProducer, error) {
	glog.V(3).Infof("attempting to setup kafka sink")
	broker, err := kafka.Dial(sinkBrokerHosts, brokerConf)
	if err != nil {
		return nil, fmt.Errorf("failed to connect to kafka cluster: %s", err)
	}
	defer broker.Close()

	//create kafka producer
	conf := kafka.NewProducerConf()
	conf.RequiredAcks = proto.RequiredAcksLocal
	producer := broker.Producer(conf)

	// create RoundRobinProducer with the default producer.
	count, err := broker.PartitionCount(topic)
	if err != nil {
		count = 1
		glog.Warningf("Failed to get partition count of topic %q: %s", topic, err)
	}
	sinkProducer := kafka.NewRoundRobinProducer(producer, count)
	glog.V(3).Infof("kafka sink setup successfully")
	return sinkProducer, nil
}
Ejemplo n.º 2
0
func (self *KafkaOutput) Init(pcf *plugins.PluginCommonConfig, conf toml.Primitive) (err error) {
	log.Println("KafkaOutput Init.")
	self.common = pcf
	hn, err := os.Hostname()
	if err != nil {
		hn = "kamanclient"
	}
	self.config = &KafkaOutputConfig{
		ClientId:      hn,
		Distributer:   "None",
		Partitions:    0,
		FlushInterval: 1000,
	}
	if err = toml.PrimitiveDecode(conf, self.config); err != nil {
		return fmt.Errorf("Can't unmarshal KafkaOutput config: %s", err)
	}
	if len(self.config.Addrs) == 0 {
		return errors.New("addrs must have at least one entry")
	}
	if len(self.config.Topic) == 0 {
		return fmt.Errorf("topic is empty")
	}

	bcf := kafka.NewBrokerConf(self.config.ClientId)
	//bcf.AllowTopicCreation = true

	// connect to kafka cluster
	self.broker, err = kafka.Dial(self.config.Addrs, bcf)
	if err != nil {
		return fmt.Errorf("cannot connect to kafka cluster: %s", err)
	}

	defer self.broker.Close()
	pf := kafka.NewProducerConf()
	pf.RequiredAcks = 1
	self.producer = self.broker.Producer(pf)
	partitions, err := self.broker.PartitionCount(self.config.Topic)
	if err != nil {
		return fmt.Errorf("cannot count to topic partitions: %s", err)
	}
	log.Printf("topic\"%s\" has %d partitions\n", self.config.Topic, partitions)
	if (self.config.Partition + 1) > partitions {
		return fmt.Errorf("invalid partition: %d, topic have %d partitions",
			self.config.Partition, partitions)
	}
	if self.config.Partitions == 0 {
		self.config.Partitions = partitions
	}
	switch self.config.Distributer {
	case "Random":
		self.distributingProducer = kafka.NewRandomProducer(self.producer, self.config.Partitions)
	case "RoundRobin":
		self.distributingProducer = kafka.NewRoundRobinProducer(self.producer, self.config.Partitions)
	case "Hash":
		self.distributingProducer = kafka.NewHashProducer(self.producer, self.config.Partitions)
	case "None":
		self.distributingProducer = nil
	default:
		return fmt.Errorf("invalid distributer: %s, must be one of these: \"Random\",\"RoundRobin\",\"Hash\"", self.config.Distributer)
	}
	self.batchChan = make(chan *outBatch)
	self.backChan = make(chan *outBatch, 2) // Never block on the hand-back
	return err
}