func newAccessLogProducer(brokerList []string) sarama.AsyncProducer {
// For the access log, we are looking for AP semantics, with high throughput.
// By creating batches of compressed messages, we reduce network I/O at a cost of more latency.
config := sarama.NewConfig()
tlsConfig := createTlsConfiguration()
if tlsConfig != nil {
config.Net.TLS.Enable = true
config.Net.TLS.Config = tlsConfig
}
config.Producer.RequiredAcks = sarama.WaitForLocal // Only wait for the leader to ack
config.Producer.Compression = sarama.CompressionSnappy // Compress messages
config.Producer.Flush.Frequency = 500 * time.Millisecond // Flush batches every 500ms
producer, err := sarama.NewAsyncProducer(brokerList, config)
if err != nil {
log.Fatalln("Failed to start Sarama producer:", err)
}
// We will just log to STDOUT if we're not able to produce messages.
// Note: messages will only be returned here after all retry attempts are exhausted.
go func() {
for err := range producer.Errors() {
log.Println("Failed to write access log entry:", err)
}
}()
return producer
}
func main() {
var addrs []string
var err error
for _, u := range strings.Split(os.Getenv("HEROKU_KAFKA_URL"), ",") {
if u, err := url.Parse(u); err == nil {
addrs = append(addrs, u.Host)
}
}
conf := sarama.NewConfig()
conf.Producer.Flush.Messages = getint("PRODUCER_FLUSH_MESSAGES", 1500)
conf.Producer.Return.Successes = true
conf.Producer.Flush.Frequency = time.Millisecond * 500
producer, err = sarama.NewAsyncProducer(addrs, conf)
if err != nil {
log.Fatal(err)
}
sarama.Logger = log.New(os.Stderr, "[Sarama] ", log.LstdFlags)
wg.Add(2)
go countSuccess()
go countErrors()
if os.Getenv("LIBRATO_TOKEN") != "" {
go librato.Librato(
metrics.DefaultRegistry,
20*time.Second,
os.Getenv("LIBRATO_OWNER"),
os.Getenv("LIBRATO_TOKEN"),
fmt.Sprintf("%s.%s", os.Getenv("LIBRATO_SOURCE"), os.Getenv("DYNO")),
[]float64{0.50, 0.95, 0.99},
time.Millisecond,
)
}
mux := httprouter.New()
mux.POST("/topics/:topic", Post)
log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", os.Getenv("PORT")), mux))
wg.Wait()
}