func Publish(input chan *FileEvent, source string, ctrl chan bool) { clientConfig := sarama.NewConfig() clientConfig.Producer.RequiredAcks = sarama.WaitForLocal clientConfig.Producer.Compression = sarama.CompressionSnappy clientConfig.Producer.Flush.Frequency = 500 * time.Millisecond clientConfig.Producer.Flush.Messages = 200 clientConfig.Producer.Flush.MaxMessages = 200 clientConfig.Producer.Flush.Bytes = 16384 clientConfig.Producer.Return.Successes = true clientConfig.Producer.Partitioner = sarama.NewRoundRobinPartitioner clientConfig.ChannelBufferSize = kafkabuffer //brokerList := []string{"127.0.0.1:9092"} var producer sarama.AsyncProducer var err error for { producer, err = sarama.NewAsyncProducer(brokerList, clientConfig) if err != nil { log.Error("Publish: Failed to start Sarama producer: ", err) log.Info("waiting....") time.Sleep(1 * time.Second) } else { break } } defer func() { if err := producer.Close(); err != nil { log.Error("Failed to shutdown producer cleanly", err) } }() registrar := &Registrar{source: source, publishCtrl: ctrl} go registrar.RegistrarDo(producer.Errors(), producer.Successes()) topic := kafkaTopic baseName := filepath.Base(source) if len(topicmap) > 0 { tmpTopic := genTopic(baseName, topicmap) if tmpTopic != "" { topic = tmpTopic } } key := hashKey for event := range input { log.Debugf("%v, %v, %v, %v", *event.Source, *event.Text, event.Line, event.Offset) key = strconv.FormatInt(event.Offset, 10) producer.Input() <- &sarama.ProducerMessage{ Topic: topic, Key: sarama.StringEncoder(key), Value: sarama.StringEncoder(*event.Text), Metadata: event, } } }