func newAccessLogProducer(brokerList []string) sarama.AsyncProducer { // For the access log, we are looking for AP semantics, with high throughput. // By creating batches of compressed messages, we reduce network I/O at a cost of more latency. config := sarama.NewConfig() tlsConfig := createTlsConfiguration() if tlsConfig != nil { config.Net.TLS.Enable = true config.Net.TLS.Config = tlsConfig } config.Producer.RequiredAcks = sarama.WaitForLocal // Only wait for the leader to ack config.Producer.Compression = sarama.CompressionSnappy // Compress messages config.Producer.Flush.Frequency = 500 * time.Millisecond // Flush batches every 500ms producer, err := sarama.NewAsyncProducer(brokerList, config) if err != nil { log.Fatalln("Failed to start Sarama producer:", err) } // We will just log to STDOUT if we're not able to produce messages. // Note: messages will only be returned here after all retry attempts are exhausted. go func() { for err := range producer.Errors() { log.Println("Failed to write access log entry:", err) } }() return producer }
func main() { var addrs []string var err error for _, u := range strings.Split(os.Getenv("HEROKU_KAFKA_URL"), ",") { if u, err := url.Parse(u); err == nil { addrs = append(addrs, u.Host) } } conf := sarama.NewConfig() conf.Producer.Flush.Messages = getint("PRODUCER_FLUSH_MESSAGES", 1500) conf.Producer.Return.Successes = true conf.Producer.Flush.Frequency = time.Millisecond * 500 producer, err = sarama.NewAsyncProducer(addrs, conf) if err != nil { log.Fatal(err) } sarama.Logger = log.New(os.Stderr, "[Sarama] ", log.LstdFlags) wg.Add(2) go countSuccess() go countErrors() if os.Getenv("LIBRATO_TOKEN") != "" { go librato.Librato( metrics.DefaultRegistry, 20*time.Second, os.Getenv("LIBRATO_OWNER"), os.Getenv("LIBRATO_TOKEN"), fmt.Sprintf("%s.%s", os.Getenv("LIBRATO_SOURCE"), os.Getenv("DYNO")), []float64{0.50, 0.95, 0.99}, time.Millisecond, ) } mux := httprouter.New() mux.POST("/topics/:topic", Post) log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", os.Getenv("PORT")), mux)) wg.Wait() }