Example #1
0
func newAccessLogProducer(brokerList []string) sarama.AsyncProducer {

	// For the access log, we are looking for AP semantics, with high throughput.
	// By creating batches of compressed messages, we reduce network I/O at a cost of more latency.
	config := sarama.NewConfig()
	tlsConfig := createTlsConfiguration()
	if tlsConfig != nil {
		config.Net.TLS.Enable = true
		config.Net.TLS.Config = tlsConfig
	}
	config.Producer.RequiredAcks = sarama.WaitForLocal       // Only wait for the leader to ack
	config.Producer.Compression = sarama.CompressionSnappy   // Compress messages
	config.Producer.Flush.Frequency = 500 * time.Millisecond // Flush batches every 500ms

	producer, err := sarama.NewAsyncProducer(brokerList, config)
	if err != nil {
		log.Fatalln("Failed to start Sarama producer:", err)
	}

	// We will just log to STDOUT if we're not able to produce messages.
	// Note: messages will only be returned here after all retry attempts are exhausted.
	go func() {
		for err := range producer.Errors() {
			log.Println("Failed to write access log entry:", err)
		}
	}()

	return producer
}
Example #2
0
File: main.go Project: cyx/faux-kr
func main() {
	var addrs []string
	var err error

	for _, u := range strings.Split(os.Getenv("HEROKU_KAFKA_URL"), ",") {
		if u, err := url.Parse(u); err == nil {
			addrs = append(addrs, u.Host)
		}
	}

	conf := sarama.NewConfig()
	conf.Producer.Flush.Messages = getint("PRODUCER_FLUSH_MESSAGES", 1500)
	conf.Producer.Return.Successes = true
	conf.Producer.Flush.Frequency = time.Millisecond * 500

	producer, err = sarama.NewAsyncProducer(addrs, conf)
	if err != nil {
		log.Fatal(err)
	}
	sarama.Logger = log.New(os.Stderr, "[Sarama] ", log.LstdFlags)

	wg.Add(2)
	go countSuccess()
	go countErrors()

	if os.Getenv("LIBRATO_TOKEN") != "" {
		go librato.Librato(
			metrics.DefaultRegistry,
			20*time.Second,
			os.Getenv("LIBRATO_OWNER"),
			os.Getenv("LIBRATO_TOKEN"),
			fmt.Sprintf("%s.%s", os.Getenv("LIBRATO_SOURCE"), os.Getenv("DYNO")),
			[]float64{0.50, 0.95, 0.99},
			time.Millisecond,
		)
	}

	mux := httprouter.New()
	mux.POST("/topics/:topic", Post)

	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", os.Getenv("PORT")), mux))
	wg.Wait()
}