func newDataCollector(brokerList []string) sarama.SyncProducer { // For the data collector, we are looking for strong consistency semantics. // Because we don't change the flush settings, sarama will try to produce messages // as fast as possible to keep latency low. config := sarama.NewConfig() config.Producer.RequiredAcks = sarama.WaitForAll // Wait for all in-sync replicas to ack the message config.Producer.Retry.Max = 10 // Retry up to 10 times to produce the message tlsConfig := createTlsConfiguration() if tlsConfig != nil { config.Net.TLS.Config = tlsConfig config.Net.TLS.Enable = true } // On the broker side, you may want to change the following settings to get // stronger consistency guarantees: // - For your broker, set `unclean.leader.election.enable` to false // - For the topic, you could increase `min.insync.replicas`. producer, err := sarama.NewSyncProducer(brokerList, config) if err != nil { log.Fatalln("Failed to start Sarama producer:", err) } return producer }
func main() { flag.Parse() if *brokerList == "" { printUsageErrorAndExit("no -brokers specified. Alternatively, set the KAFKA_PEERS environment variable") } if *topic == "" { printUsageErrorAndExit("no -topic specified") } if *verbose { sarama.Logger = logger } config := sarama.NewConfig() config.Producer.RequiredAcks = sarama.WaitForAll switch *partitioner { case "": if *partition >= 0 { config.Producer.Partitioner = sarama.NewManualPartitioner } else { config.Producer.Partitioner = sarama.NewHashPartitioner } case "hash": config.Producer.Partitioner = sarama.NewHashPartitioner case "random": config.Producer.Partitioner = sarama.NewRandomPartitioner case "manual": config.Producer.Partitioner = sarama.NewManualPartitioner if *partition == -1 { printUsageErrorAndExit("-partition is required when partitioning manually") } default: printUsageErrorAndExit(fmt.Sprintf("Partitioner %s not supported.", *partitioner)) } message := &sarama.ProducerMessage{Topic: *topic, Partition: int32(*partition)} if *key != "" { message.Key = sarama.StringEncoder(*key) } if *value != "" { message.Value = sarama.StringEncoder(*value) } else if stdinAvailable() { bytes, err := ioutil.ReadAll(os.Stdin) if err != nil { printErrorAndExit(66, "Failed to read data from the standard input: %s", err) } message.Value = sarama.ByteEncoder(bytes) } else { printUsageErrorAndExit("-value is required, or you have to provide the value on stdin") } producer, err := sarama.NewSyncProducer(strings.Split(*brokerList, ","), config) if err != nil { printErrorAndExit(69, "Failed to open Kafka producer: %s", err) } defer func() { if err := producer.Close(); err != nil { logger.Println("Failed to close Kafka producer cleanly:", err) } }() partition, offset, err := producer.SendMessage(message) if err != nil { printErrorAndExit(69, "Failed to produce message: %s", err) } else if !*silent { fmt.Printf("topic=%s\tpartition=%d\toffset=%d\n", *topic, partition, offset) } }